import urllib.request
import time
import csv

# 主函数
def main():
    print("当当网爬虫开始")
    print("开始时间: " + time.strftime('%Y-%m-%d %H:%M:%S'))
    
    # 存储所有书籍信息，使用集合去重
    all_books = []
    book_titles = set()  # 用于去重
    max_pages = 25  # 500本书，每页约20本，需要25页
    max_books = 500  # 最多爬取500本书
    
    # 设置请求头
    headers = {}
    headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
    headers['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
    headers['Accept-Language'] = 'zh-CN,zh;q=0.8'
    
    try:
        # 循环爬取多页
        for page_num in range(1, max_pages + 1):
            # 如果已经爬够500本，就停止
            if len(all_books) >= max_books:
                print("\n已达到最大爬取数量" + str(max_books) + "本，停止爬取")
                break
            
            url = 'http://bang.dangdang.com/books/bestsellers/01.00.00.00.00.00-24hours-0-0-1-' + str(page_num)
            print("\n=== 正在爬取第" + str(page_num) + "页 ===")
            print("URL: " + url)
            
            # 创建请求
            req = urllib.request.Request(url, headers=headers)
            
            # 发送请求
            try:
                with urllib.request.urlopen(req, timeout=15) as response:
                    print("状态码: " + str(response.status))
                    
                    # 读取内容
                    content = response.read()
                    
                    # 尝试解码内容
                    try:
                        html = content.decode('gbk')
                    except UnicodeDecodeError:
                        html = content.decode('utf-8', 'ignore')
                    
                    # 保存第1页内容用于调试
                    if page_num == 1:
                        with open('page_content.html', 'w', encoding='utf-8') as f:
                            f.write(html)
                        print("页面内容已保存到 page_content.html")
                    
                    # 提取书籍信息
                    lines = html.split('\n')
                    page_books_count = 0
                    
                    # 遍历HTML行，查找书籍信息
                    i = 0
                    while i < len(lines) and len(all_books) < max_books:
                        line = lines[i]
                        
                        # 查找书籍条目开始的标记
                        if '<li>' in line and '<div class="list_num' in lines[i+1]:
                            # 提取排名
                            rank = ''
                            if i+1 < len(lines):
                                rank_line = lines[i+1]
                                if 'list_num' in rank_line:
                                    # 提取数字
                                    for char in rank_line:
                                        if char.isdigit():
                                            rank += char
                            
                            # 提取书名
                            title = '未知'
                            j = i
                            while j < i + 10 and j < len(lines):
                                if '<div class="name">' in lines[j] and '<a href=' in lines[j]:
                                    # 查找title属性
                                    if 'title="' in lines[j]:
                                        parts = lines[j].split('title="')
                                        if len(parts) > 1:
                                            title_part = parts[1].split('"')[0]
                                            title = title_part
                                            break
                                j += 1
                            
                            # 只有当标题有效且未重复时才继续提取其他信息
                            if title != '未知' and len(title) > 5 and title not in book_titles:
                                # 提取作者
                                author = '未知'
                                
                                # 方法1：查找publisher_info区域附近的作者信息
                                j = i
                                while j < i + 30 and j < len(lines):
                                    if '<div class="publisher_info">' in lines[j]:
                                        # 向前和向后搜索更大的范围
                                        for k in range(max(0, j - 5), min(j + 15, len(lines))):
                                            # 情况1：查找(作者:...)格式
                                            if '(' in lines[k] and '作者:' in lines[k] and ')' in lines[k]:
                                                start = lines[k].find('作者:') + 3
                                                end = lines[k].find(')', start)
                                                if start < end:
                                                    author = lines[k][start:end].strip()
                                                    break
                                            # 情况2：查找明确的作者:标记
                                            elif '作者:' in lines[k]:
                                                author_start = lines[k].find('作者:') + 3
                                                author_end = lines[k].find('<', author_start)
                                                if author_end == -1:
                                                    author_end = len(lines[k])
                                                author = lines[k][author_start:author_end].strip()
                                                break
                                            # 情况3：查找包含译者信息的格式
                                            elif ('译' in lines[k] or '译者' in lines[k]) and author == '未知':
                                                # 清理HTML标签
                                                clean_text = ''
                                                in_tag = False
                                                for char in lines[k]:
                                                    if char == '<':
                                                        in_tag = True
                                                    elif char == '>':
                                                        in_tag = False
                                                    elif not in_tag:
                                                        clean_text += char
                                                # 提取可能的作者信息
                                                if clean_text.strip() and len(clean_text.strip()) < 100:
                                                    author = clean_text.strip()
                                        if author != '未知':
                                            break
                                    j += 1
                                
                                # 方法2：如果方法1失败，尝试从整个书籍条目中提取
                                if author == '未知':
                                    # 在当前<li>标签范围内搜索
                                    k = i
                                    while k < i + 100 and k < len(lines):
                                        # 查找特定的作者链接格式
                                        if 'http://author.dangdang.com/' in lines[k] and '<a href=' in lines[k]:
                                            # 从链接中提取作者名
                                            if '>' in lines[k] and '</a>' in lines[k]:
                                                a_start = lines[k].find('>') + 1
                                                a_end = lines[k].find('</a>')
                                                if a_start < a_end:
                                                    author = lines[k][a_start:a_end].strip()
                                                    break
                                        # 查找包含作者特征的文本
                                        has_author_markers = ('著' in lines[k] or '编' in lines[k] or '编著' in lines[k] or '译' in lines[k] or '绘' in lines[k])
                                        no_publisher_info = ('出版社' not in lines[k] and '出版时间' not in lines[k])
                                        if has_author_markers and no_publisher_info:
                                            # 清理HTML标签
                                            clean_text = ''
                                            in_tag = False
                                            for char in lines[k]:
                                                if char == '<':
                                                    in_tag = True
                                                elif char == '>':
                                                    in_tag = False
                                                elif not in_tag:
                                                    clean_text += char
                                            # 简单验证是否为作者信息
                                            clean_text = clean_text.strip()
                                            # 避免把书名误当作者
                                            if clean_text and len(clean_text) < 150 and not clean_text.startswith('20') and not clean_text.startswith('19'):
                                                # 检查是否包含书名的主要部分
                                                contains_title = False
                                                title_parts = title.split()[:2]  # 取书名的前两个词
                                                for part in title_parts:
                                                    if len(part) > 2 and part in clean_text:
                                                        contains_title = True
                                                        break
                                                if not contains_title:
                                                    author = clean_text
                                                    break
                                        k += 1
                                
                                # 方法3：特殊处理知名书籍的作者
                                special_books = {
                                    '悉达多': '[德]赫尔曼·黑塞',
                                    '活着': '余华',
                                    '被讨厌的勇气': '[日]岸见一郎 古贺史健',
                                    '明朝那些事儿': '当年明月',
                                    '大中华寻宝记': '孙家裕',
                                    '掌控习惯': '[美]詹姆斯·克利尔',
                                    '窄门': '[法]安德烈·纪德'
                                }
                                for book_key, known_author in special_books.items():
                                    if book_key in title and author == '未知':
                                        author = known_author
                                 
                                # 清理作者字段
                                if author != '未知':
                                    # 移除年份前缀
                                    if len(author) >= 4 and author[:4].isdigit() and (author[4:5] == '-' or author[4:5] == '/'):
                                        # 按空格分割，保留非日期部分
                                        parts = author.split()
                                        clean_author = ''
                                        for part in parts:
                                            if not (len(part) >= 4 and part[:4].isdigit() and (part[4:5] == '-' or part[4:5] == '/')):
                                                clean_author += part + ' '
                                        author = clean_author.strip()
                                        
                                    # 先处理分号、逗号分隔的情况
                                    if any(sep in author for sep in ['；', '，', ',', '、']):
                                        for sep in ['；', '，', ',', '、']:
                                            if sep in author:
                                                parts = author.split(sep)
                                                # 查找包含作者标记的部分
                                                for part in parts:
                                                    if any(marker in part for marker in ['著', '编', '译', '绘', '作']):
                                                        author = part.strip()
                                                        break
                                                # 如果没有作者标记，取第一个部分
                                                if '著' not in author and '编' not in author and '译' not in author:
                                                    author = parts[0].strip()
                                                break
                                    
                                    # 移除出版信息和其他无关内容
                                    keywords = ['出版社', '出版公司', '出版集团', '年', '月', '日', '当当', '自营', '正版', '现货', '套装', '绘本', '漫画']
                                    for keyword in keywords:
                                        if keyword in author:
                                            # 简单分割
                                            parts = author.split(keyword)
                                            author = parts[0].strip()
                                            # 递归处理，确保移除所有关键词
                                            if keyword in author:
                                                parts = author.split(keyword)
                                                author = parts[0].strip()
                                            break
                                    
                                    # 移除多余的标点和特殊字符
                                    author = author.replace('&nbsp;', '').strip()
                                    # 移除末尾的标点符号
                                    while author and author[-1] in '.,;；，。、':
                                        author = author[:-1].strip()
                                    
                                    # 验证作者文本的合理性
                                    if not author or len(author) < 2 or author.isdigit() or len(author) > 50:
                                        author = '未知'
                                
                                # 提取出版时间和出版社
                                pub_time = '未知'
                                publisher = '未知'
                                # 查找包含出版信息的行
                                j = i
                                while j < i + 50 and j < len(lines):
                                    # 查找publisher_info区域，这是包含作者、出版社、出版时间的主要区域
                                    if '<div class="publisher_info">' in lines[j]:
                                        # 从publisher_info区域开始搜索接下来的几行
                                        for k in range(j, min(j + 10, len(lines))):
                                            # 首先尝试提取出版时间（通常在<span>标签内）
                                            if pub_time == '未知' and '<span>' in lines[k] and '</span>' in lines[k]:
                                                # 检查是否包含年份格式 YYYY-MM-DD
                                                time_part = lines[k].split('<span>')[1].split('</span>')[0].strip()
                                                # 简单的年份格式验证
                                                if (len(time_part) >= 4 and time_part[:4].isdigit() and 
                                                    (time_part[4:5] == '-' or time_part[4:5] == '/')):
                                                    pub_time = time_part
                                            
                                            # 提取出版社 - 方法1: 查找特定的出版社链接
                                            if publisher == '未知' and 'http://category.dangdang.com/pg' in lines[k]:
                                                # 查找title属性
                                                if 'title="' in lines[k]:
                                                    title_start = lines[k].find('title="')
                                                    if title_start > 0:
                                                        title_end = lines[k].find('"', title_start + 7)
                                                        if title_end > 0:
                                                            pub_text = lines[k][title_start + 7:title_end].strip()
                                                            # 过滤条件：包含出版社关键词
                                                            if ('出版社' in pub_text or '出版公司' in pub_text or '出版集团' in pub_text):
                                                                publisher = pub_text
                                                # 从标签内容提取
                                                if publisher == '未知':
                                                    a_start = lines[k].find('>')
                                                    a_end = lines[k].find('</a>')
                                                    if a_start > 0 and a_end > a_start:
                                                        pub_text = lines[k][a_start + 1:a_end].strip()
                                                        if ('出版社' in pub_text or '出版公司' in pub_text or '出版集团' in pub_text):
                                                            publisher = pub_text
                                            
                                            # 提取出版社 - 方法2: 查找包含出版社关键词的文本
                                            if publisher == '未知':
                                                # 清理HTML标签
                                                clean_text = ''
                                                in_tag = False
                                                for char in lines[k]:
                                                    if char == '<':
                                                        in_tag = True
                                                    elif char == '>':
                                                        in_tag = False
                                                    elif not in_tag:
                                                        clean_text += char
                                                # 从清理后的文本中提取出版社
                                                if '出版社' in clean_text or '出版公司' in clean_text:
                                                    # 简单分割并查找包含出版社的部分
                                                    parts = clean_text.split('|')  # 有些格式使用|分隔
                                                    if len(parts) > 1:
                                                        for part in parts:
                                                            if ('出版社' in part or '出版公司' in part or '出版集团' in part):
                                                                publisher = part.strip()
                                                                break
                                                    # 如果没有|，尝试直接查找
                                                    if publisher == '未知':
                                                        words = clean_text.split()
                                                        for word in words:
                                                            if ('出版社' in word or '出版公司' in word or '出版集团' in word) and len(word) < 50:
                                                                publisher = word
                                                                break
                    
                                    j += 1
                                
                                # 清理可能的格式问题 - 处理包含&nbsp;混合内容
                                if '&nbsp;' in publisher:
                                    # 分离出正确的出版社信息
                                    parts = publisher.split('&nbsp;')
                                    for part in parts:
                                        if ('出版社' in part or '出版公司' in part or '出版集团' in part):
                                            publisher = part
                                            break
                                    # 如果没有找到明显的出版社，取最后一部分
                                    if ('出版社' not in publisher and '出版公司' not in publisher and '出版集团' not in publisher) and len(parts) > 0:
                                        publisher = parts[-1]
                                
                                # 提取评论数
                                comments = '未知'
                                j = i
                                while j < i + 20 and j < len(lines):
                                    if '<div class="star">' in lines[j]:
                                        # 查找包含评论数的<a>标签或<span>标签
                                        k = j
                                        while k < j + 5 and k < len(lines):
                                            # 查找评论数的模式
                                            if '条评论' in lines[k]:
                                                # 从后往前找，找到数字部分
                                                comment_line = lines[k]
                                                temp_comments = ''
                                                # 只提取条评论前面的数字
                                                before_comments = comment_line.split('条评论')[0]
                                                # 从末尾向前查找连续的数字
                                                for char in reversed(before_comments.strip()):
                                                    if char.isdigit():
                                                        temp_comments = char + temp_comments
                                                    else:
                                                        break
                                                if temp_comments and len(temp_comments) <= 10:  # 限制长度避免异常数字
                                                    comments = temp_comments
                                                    break
                                            k += 1
                                        break
                                    j += 1
                                
                                # 提取价格
                                price = '未知'
                                j = i
                                while j < i + 30 and j < len(lines):
                                    if '<span class="price_n">' in lines[j]:
                                        price_line = lines[j]
                                        # 提取价格数字
                                        temp_price = ''
                                        if '<span class="price_n">' in price_line:
                                            parts = price_line.split('<span class="price_n">')
                                            if len(parts) > 1:
                                                price_part = parts[1]
                                                for char in price_part:
                                                    if char.isdigit() or char == '.':
                                                        temp_price += char
                                                    elif temp_price:
                                                        break
                                        if temp_price:
                                            price = temp_price
                                            break
                                    j += 1
                                
                                # 添加到书籍列表
                                book = {
                                    '排名': rank if rank else str(len(all_books) + 1),
                                    '书名': title,
                                    '作者': author,
                                    '出版社': publisher,
                                    '出版时间': pub_time,
                                    '评论数': comments,
                                    '价格': price
                                }
                                
                                all_books.append(book)
                                book_titles.add(title)
                                page_books_count += 1
                                print(str(len(all_books)) + ". " + title + " (作者: " + author + ", 价格: " + price + ", 评论: " + comments + "条")
                            
                            # 跳过当前书籍条目
                            j = i
                            while j < len(lines) and '</li>' not in lines[j]:
                                j += 1
                            i = j
                        else:
                            i += 1
                    
                    print("第" + str(page_num) + "页爬取完成，新增" + str(page_books_count) + "本书")
                    
                    # 延迟，避免请求过快
                    if page_num < max_pages and len(all_books) < max_books:
                        delay = 2  # 固定延迟2秒
                        print("休息" + str(delay) + "秒后继续...")
                        time.sleep(delay)
                    
            except Exception as e:
                print("爬取第" + str(page_num) + "页时出错: " + str(e))
                # 出错时继续下一页
                continue
        
        # 保存到CSV
        if all_books:
            csv_file = 'dangdang_bestsellers_500.csv'
            with open(csv_file, 'w', newline='', encoding='utf-8-sig') as f:
                writer = csv.writer(f)
                writer.writerow(['排名', '书名', '作者', '出版社', '出版时间', '评论数', '价格'])
                for book in all_books:
                    writer.writerow([
                        book['排名'], 
                        book['书名'], 
                        book['作者'], 
                        book['出版社'],
                        book['出版时间'],
                        book['评论数'],
                        book['价格']
                    ])
            print("\n🎉 爬取完成！")
            print("成功保存 " + str(len(all_books)) + " 本书籍信息到 " + csv_file)
        else:
            print("\n未找到书籍信息")
    
    except Exception as e:
        print("发生严重错误: " + str(e))
        # 保存已爬取的数据
        if all_books:
            csv_file = 'dangdang_bestsellers_partial.csv'
            with open(csv_file, 'w', newline='', encoding='utf-8-sig') as f:
                writer = csv.writer(f)
                writer.writerow(['排名', '书名', '作者', '出版社', '出版时间', '评论数', '价格'])
                for book in all_books:
                    writer.writerow([
                        book['排名'], 
                        book['书名'], 
                        book['作者'], 
                        book['出版社'],
                        book['出版时间'],
                        book['评论数'],
                        book['价格']
                    ])
            print("已保存部分数据到 " + csv_file)
        
        # 保存错误信息
        with open('error_log.txt', 'w') as f:
            f.write("错误: " + str(e))
    
    print("\n结束时间: " + time.strftime('%Y-%m-%d %H:%M:%S'))
    print("爬虫结束")

if __name__ == "__main__":
    main()